#!/usr/bin/python
# -*-coding:utf-8-*-
import os
import pandas as pd

from zg_data_process.zg_data_process import DataProcess

data_process_api = DataProcess()

demo_dir = '/db/zg_data/zbc/jh_fund_cooperative_project/linear_dynamic_style_alloc/csi500_corr_synthesis_model/cache'
demo_filename = 'ths_concat_and_label_synthesis_factor_on_csi500_pool_data'

demo_data_cache_dir = '/db/zg_data/zbc/buffer'
demo_data_cache_filename = 'zg02_data_process_demo_data_1'

is_cache_data = False

factor_name_list = [
    'csi500_cashflow1_syn_factor_value',
    'csi500_cashflow2_syn_factor_value',
    'csi500_growth1_syn_factor_value',
    'csi500_growth2_syn_factor_value',
    'csi500_hf1_syn_factor_value',
    'csi500_hf2_syn_factor_value',
    'csi500_liquidity1_syn_factor_value',
    'csi500_liquidity2_syn_factor_value',
    'csi500_liquidity3_syn_factor_value',
    'csi500_mom1_syn_factor_value',
    'csi500_mom2_syn_factor_value',
    'csi500_mom3_syn_factor_value',
    'csi500_mom4_syn_factor_value',
    'csi500_network_syn_factor_value',
    'csi500_other1_syn_factor_value',
    'csi500_other2_syn_factor_value',
    'csi500_profitability1_syn_factor_value',
    'csi500_profitability2_syn_factor_value',
    'csi500_profitability3_syn_factor_value',
    'csi500_profitability4_syn_factor_value',
    'csi500_profitability5_syn_factor_value',
    'csi500_profitability6_syn_factor_value',
    'csi500_quality_syn_factor_value',
    'csi500_rvolatility1_syn_factor_value',
    'csi500_rvolatility2_syn_factor_value',
    'csi500_valuation1_syn_factor_value',
    'csi500_valuation2_syn_factor_value',
    'csi500_valuation3_syn_factor_value',
    'csi500_volatility1_syn_factor_value',
    'csi500_volatility2_syn_factor_value',
    'csi500_volatility3_syn_factor_value',
    'csi500_volatility4_syn_factor_value',
]


if is_cache_data:
    concat_factor_data = pd.read_hdf(os.path.join(demo_dir, demo_filename + '.h5'))

    selected_date = [
        '2015-07-09',
        '2015-07-10',
        '2015-07-11',
    ]

    demo_data = concat_factor_data[concat_factor_data['date'].isin(selected_date)].copy()

    demo_data.to_hdf(os.path.join(demo_data_cache_dir, demo_data_cache_filename+'.h5'),
                     key=demo_data_cache_filename)
else:
    demo_data = pd.read_hdf(os.path.join(demo_data_cache_dir, demo_data_cache_filename+'.h5'))


# TODO - 数据过滤
# 去掉delist_date为nan的
demo_data = demo_data[~demo_data['delist_date'].isnull()]
print('concat factor data drop delist date nan done!')

# 过滤退市股票
demo_data = demo_data[demo_data['date'] < demo_data['delist_date']]
demo_data = demo_data[~demo_data['scale_total_market_size'].isnull()]    # 一般市值为空都是退市股票
print('concat factor data drop delist stock and null cap data done!')

# 去掉st
demo_data = data_process_api.st_filteration(df=demo_data,
                                            drop_label=True)
print('concat factor data drop st and new&sub-new stocks done!')

# TODO - 去掉空值的申万一级行业
print('before drop sw1 nan code, shape is', demo_data.shape)
demo_data = demo_data[~demo_data['sw1_code'].isnull()]
print('after drop sw1 nan code, shape is', demo_data.shape)

## TODO - 因子基本处理
# 1. 缺失值处理
print('before drop nan factor data shape is', demo_data.shape)
# demo_data = demo_data.dropna()
# demo_data = data_process_api.filled_with_sw1_stats_value(df=demo_data,
demo_data = data_process_api.filled_with_market_stats_value(df=demo_data,
                                                            columns=factor_name_list,
                                                            q=0.5)
print('after drop nan factor data shape is', demo_data.dropna().shape)
print('concat factor data sw1 fill nan done!')

# 2. 去极值
demo_data = data_process_api.cs_mad_outlier_process(df=demo_data,
                                                    columns=factor_name_list,
                                                    drop=False,
                                                    copy=True,
                                                    verbose=False)
print('concat factor data mad outlier processed!')

# 3. 标准化
demo_data = data_process_api.cs_z_score_normalization_process(df=demo_data,
                                                             columns=factor_name_list,
                                                             copy=True,
                                                             verbose=False)
print('concat factor data normalized!')

# 4. 市值中性化
demo_data = data_process_api.cs_cap_neutral_process(df=demo_data,
                                                   columns=factor_name_list)
print('concat factor data cap neutralization done!')

# # 4. 行业和市值中性化
# demo_data = data_process_api.cs_cap_sw1_ind_neutral_process(df=demo_data,
#                                                             factor_columns=factor_name_list)
# print('concat factor data cap and sw1-ind neutralization done!')





